import os.path
import pickle
from multiprocessing import Process
from concurrent.futures import ThreadPoolExecutor
from lxml import etree
from multiprocessing import Process, Queue

import requests

'''
多进程采集美女图库
分析流程：
1、从页面中解析出详情页的url，从详情页提取图片下载地址（进程1）
2、用队列，把进程1得到的图片地址传给进程2，以供下载。注意有2种方法获取队列，目前只有②能办到
   ①import queue.Queue
   ②from multiprocessing import Process, Queue


3、把拿到的下载地址进行下载图片（进程2）

'''

headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/114.0'}
path = '../thread-demo/images'
thread_num = 10

# 进程1，获取图片地址
def get_img_src(q):
    url = 'https://www.jxmm.net/'
    response = requests.get(url, headers=headers)
    response.encoding = 'utf-8'
    html = response.text

    # 用xpath获取详细页地址
    et = etree.HTML(html)
    result = et.xpath('//div[@class="pic"]/ul/li/a[1]')
    if result:
        for r in result:
            detail_url = url + r.get('href')

            # 从详细页获取图片地址
            child_response = requests.get(detail_url, headers=headers)
            child_response.encoding = 'utf-8'
            child_html = child_response.text

            et_child = etree.HTML(child_html)
            src = et_child.xpath('//div[@class="content"]/a/img/@src')[0]

            # 把图片地址装入队列
            q.put(src)

        # 当进程1结束后，设置一个结束符，否则进程2不知道什么时候进程1结束
        q.put('over')

# 进程2，下载图片
def download_img(q):
    # 从队列获取图片链接地址，并开启2个线程，如果没有数据就会阻塞
    with ThreadPoolExecutor(thread_num) as t:
        while True:
            src = q.get()
            if src == 'over':
                break
            else:
                t.submit(download, src)

def download(img_url):
    # 图片名
    file_name = os.path.basename(img_url)

    # 本地生成图片路径
    file_path = path + '/' + file_name

    img = requests.get(img_url).content

    # 图片路径
    with open(file_path, mode='wb') as f:
        f.write(img)
    print('下载图片：{} 生成图片：{}'.format(img_url, file_path))




if __name__ == '__main__':
    if not os.path.exists(path):
        os.mkdir(path)

    # 造一个队列，用于进程1传输详情页地址给进程2
    q = Queue()

    # 进程1，获取图片地址
    p1 = Process(target=get_img_src, args=(q,))

    # 进程2，下载图片
    p2 = Process(target=download_img, args=(q,))

    p1.start()
    p2.start()